from db_class.hbaseClass import HBaseClass
from db_class.hdfsClass import HDFSClass
from db_class.mongdbClass import MongdbClass
from settings import Config
from sfID_client.sfID_client import SfIDClient

if __name__ == "__main__":
    # 配置项
    config = Config()
    
    # 雪花算法
    sf = SfIDClient(config.SNOWFLAKE_HOST, config.SNOWFLAKE_PORT)
    
    # 获取数据
    mongo = MongdbClass(config.REPTILE_HOST, config.REPTILE_DB, config.REPTILE_COL)
    # mongo = MongdbClass(config.MONGO_HOST, config.MONGO_DB, config.MONGO_COL)
    # mongo = MongdbClass(config.REPTILE_HOST, "2023", "08")

    data = []
    for i in mongo.findAll():
        new_data = {}
        new_data['id'] = str(sf.getID())
        new_data['year'] = int(i['year'])
        new_data['month'] = int(i['month'])
        new_data['date'] = int(i['date'].split("-")[1])
        new_data['_id'] = i['_id']
        new_data['title'] = i['title']
        new_data['type'] = i['type']
        new_data['url'] = i['url']
        new_data['time'] = i['time']
        data.append(new_data)

    # 备份
    backups = MongdbClass(config.MONGO_HOST, config.MONGO_DB, config.MONGO_COL)
    backups.addMany(data)

    # 导入hbase
    HBasedb = HBaseClass(config.HBASE_HOST, config.HBASE_PORT)
    HBasedb.addDate(data)

    # 导入hdfs
    HDFSdb = HDFSClass(config.HDFS_HOST, ("title", "type", "time", "url", "date", "month", "year"))
    HDFSdb.writeFile(data)
    HDFSdb.uploadFile()

    # 清除已导入的数据
    mongo.delAll()
